# -*- coding: utf-8 -*-
import os
import pymongo
from base64 import b64encode
import requests
import lxml.html as H
from html2text import html2text




def move_pictures():
    #pic_dir = "pictures\\full"
    pic_dir = "pictures\\thumbs\\big"
    pics = [ pic for pic in os.listdir(pic_dir) 
            if pic.endswith(".jpg")]
    for pic in pics:
        sub_dir = pic[:2]
        new_dir = os.path.join(pic_dir, sub_dir)
        if not os.path.exists(new_dir):
            os.mkdir(new_dir)
        old_path = os.path.join(pic_dir, pic)
        new_path = os.path.join(pic_dir, os.path.join(sub_dir, pic))
        try:
            os.rename(old_path, new_path)
        except:
            if os.path.exists(new_path):
                os.remove(old_path)

def update_data():
    connection = pymongo.Connection('localhost', 27017)
    db = connection['douban_group']
    collection = db['haixiuzu']
    i = 0
    for one in collection.find():
        #i = i + 1
        #if i > 3:break
        update_flag = False
        _id = one['_id']
        print _id
        images = one['images']
        #print '-' * 80
        #print images
        for x in range(len(images)):
            path = images[x]['path']
            if not os.path.exists(os.path.join('pictures', path)):
                update_flag = True
                sub_dir = path.split('/')[1][:2]
                file_name = path.split('/')[1]
                new_path = "full/%s/%s" % (sub_dir, file_name)
                images[x]['path'] = new_path
        #print '-' * 80
        #print images
        if update_flag:
            print "-----------update------------"
            collection.update({"_id":_id}, {"$set":{"images":images}})
                
def check_empty_images():
    connection = pymongo.Connection('localhost', 27017)
    db = connection['douban_group']
    collection = db['haixiuzu']
    for one in collection.find():
        _id = one['_id']
        if not one.has_key("images"):
            print "not has image keys:", _id
            #collection.remove({"_id":_id})
        else:
            images = one['images']
            if len(images) == 0:
                print "need to download again:", _id


def get_author_loc(author_url):
    loc = "Uknown"
    headers = {
        'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding':'gzip, deflate, sdch',
        'Accept-Language':'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control':'max-age=0',
        'Connection':'keep-alive',
        'Cookie':'bid="X4ecWl+CdlA"; ps=y; ct=y; _ga=GA1.2.2050618562.1425631883; ll="108088"; __utma=30149280.2050618562.1425631883.1426842541.1426849181.28; __utmz=30149280.1426849181.28.6.utmcsr=bing|utmccn=(organic)|utmcmd=organic|utmctr=scrapy%20%E6%80%A7%E8%83%BD; __utmv=30149280.4169; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1426946355%2C%22http%3A%2F%2Fcn.bing.com%2Fsearch%3Fq%3Dpymongo%2B%25E6%259F%25A5%25E8%25AF%25A2%25E4%25BF%25AE%25E6%2594%25B9%26go%3D%25E6%258F%2590%25E4%25BA%25A4%26qs%3Dn%26form%3DQBLH%26pq%3Dpymongo%2B%25E6%259F%25A5%25E8%25AF%25A2%25E4%25BF%25AE%25E6%2594%25B9%26sc%3D0-8%26sp%3D-1%26sk%3D%26cvid%3Dcb6989f3d5004d61909cc7456ec6ff1c%22%5D; ap=1; ue="faker@163.com"; dbcl2="4529591:QS8Wsq64kdY"; ck="YRIs"; push_noty_num=0; push_doumail_num=0; _pk_id.100001.8cb4=65c8da152f14be8a.1425628829.35.1426950090.1426940466.; _pk_ses.100001.8cb4=*',
        'Host':'www.douban.com',
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.76 Safari/537.36'
    }

    response = requests.get(author_url, headers=headers)
    if not response.ok:
        return "Error"
    doc = H.document_fromstring(response.text)
    loc_html = doc.xpath('//div[@class="mod user-card"]/div[2]/ul[1]/li[2]')[0].text_content()
    loc = html2text(loc_html).strip('\r\n\t ')
    if loc == "":
        loc = "Uknown"
    else:
        loc = html2text(loc_html).strip('\n ')
    if loc != "Uknown":
        print "got :", author_url
    return loc

def check_loc():
    connection = pymongo.Connection('localhost', 27017)
    db = connection['douban_group']
    collection = db['haixiuzu']
    cur = collection.find({"loc":{"$exists":0}})
    for one in cur:
        _id = one['_id']
        loc = get_author_loc(one['author_url'])
        if loc == "Error":
            continue
        collection.update({"_id":_id}, {"$set":{"loc":loc}})


if __name__ == '__main__':
    #get_author_loc("http://www.douban.com/group/people/mktime/")
    #check_empty_images()
    check_loc()
